From b71ef711e6beb712aa8f8e2099ad5b065c65ce15 Mon Sep 17 00:00:00 2001 From: Ralf Horstmann Date: Fri, 14 Feb 2020 21:09:38 +0100 Subject: [PATCH] Implement GCID parsing in unicsv and extend ID to 64bit (#501) * Implement GCID parsing in unicsv and extend ID to 64bit This reimplements the parser and adds some test cases. The test cases have been validated with several online converters, so this hopefully works better than the previous implementation. This also fixes an endless loop that afl managed to trigger. * Fix clazy warning in unicsv * Add GC-ID test cases, documentation, int parsing - Explicitly do the integer parsing with base 10, otherwise hex values and octal values would be accepted with 0x and 0 prefix. - Add testcases that cover GC-ID parsing, including some invalid cases - Unify the terminology (GC-Code vs GC-ID) - Add examples to the unicsv documentation showing that both GC-ID and GC-Code are accepted --- defs.h | 2 +- gpx.cc | 2 +- reference/gc/gcid-input.csv | 57 +++++++++++++++++++++++++++ reference/gc/gcid-output.csv | 57 +++++++++++++++++++++++++++ testo.d/unicsv.test | 3 ++ unicsv.cc | 76 ++++++++++++++++++++++++++++-------- xmldoc/formats/unicsv.xml | 2 +- 7 files changed, 179 insertions(+), 20 deletions(-) create mode 100644 reference/gc/gcid-input.csv create mode 100644 reference/gc/gcid-output.csv diff --git a/defs.h b/defs.h index b1a9fc486..6cd0bd00f 100644 --- a/defs.h +++ b/defs.h @@ -286,7 +286,7 @@ public: placer_id(0), favorite_points(0) {} - int id; /* The decimal cache number */ + long long id; /* The decimal cache number */ geocache_type type:5; geocache_container container:4; unsigned int diff:6; /* (multiplied by ten internally) */ diff --git a/gpx.cc b/gpx.cc index 8c19589a3..cb1be39b6 100644 --- a/gpx.cc +++ b/gpx.cc @@ -207,7 +207,7 @@ GpxFormat::tag_gs_cache(const QXmlStreamAttributes& attr) geocache_data* gc_data = wpt_tmp->AllocGCData(); if (attr.hasAttribute("id")) { - gc_data->id = attr.value("id").toString().toInt(); + gc_data->id = attr.value("id").toString().toLongLong(); } if (attr.hasAttribute("available")) { if (attr.value("available").compare(QLatin1String("True"), Qt::CaseInsensitive) == 0) { diff --git a/reference/gc/gcid-input.csv b/reference/gc/gcid-input.csv new file mode 100644 index 000000000..d4d81a1b6 --- /dev/null +++ b/reference/gc/gcid-input.csv @@ -0,0 +1,57 @@ +lat,lon,gcid +1.0, 2.0, GC1 +1.0, 2.0, GC01 +1.0, 2.0, GC001 +1.0, 2.0, GC0001 +1.0, 2.0, GC0010 +1.0, 2.0, GC0100 +1.0, 2.0, GC1000 +1.0, 2.0, GCS000 +1.0, 2.0, GCFFFF +1.0, 2.0, GC0G000 +1.0, 2.0, GC10000 +1.0, 2.0, GCG0000 +1.0, 2.0, GCZ000O +1.0, 2.0, GCZ000S +1.0, 2.0, GCZZZZZ +1.0, 2.0, GCZZZZZZ +1.0, 2.0, GCZZZZZZZ +1.0, 2.0, GCZZZZZZZZ +1.0, 2.0, GCZZZZZZZZZ +1.0, 2.0, GCZZZZZZZZZZ +1.0, 2.0, GCZZZZZZZZZZZ +1.0, 2.0, GCZZZZZZZZZZZZ +1.0, 2.0, GC0 +1.0, 2.0, GC00 +1.0, 2.0, GC000 +1.0, 2.0, GC0000 +1.0, 2.0, GC0G00 +1.0, 2.0, GC0H00 +1.0, 2.0, GC0I00 +1.0, 2.0, GC0J00 +1.0, 2.0, GC0K00 +1.0, 2.0, GC0L00 +1.0, 2.0, GC0M00 +1.0, 2.0, GC0O00 +1.0, 2.0, GC0P00 +1.0, 2.0, GC0Q00 +1.0, 2.0, GC0R00 +1.0, 2.0, GC0T00 +1.0, 2.0, GC0U00 +1.0, 2.0, GC0V00 +1.0, 2.0, GC0W00 +1.0, 2.0, GC0X00 +1.0, 2.0, GC0Y00 +1.0, 2.0, GC0Z00 +1.0, 2.0, GCZ000I +1.0, 2.0, GCZ000L +1.0, 2.0, GCZ000U +1.0, 2.0, GCZZZZZZZZZZZZZ +1.0, 2.0, ABCDEF +1.0, 2.0, 0x1234 +1.0, 2.0, 12F456 +1.0, 2.0, 12.456 +1.0, 2.0, 12 456 +1.0, 2.0, 012345 +1.0, 2.0, 123456 +1.0, 2.0, 787662783788138640 diff --git a/reference/gc/gcid-output.csv b/reference/gc/gcid-output.csv new file mode 100644 index 000000000..0dc67c5b9 --- /dev/null +++ b/reference/gc/gcid-output.csv @@ -0,0 +1,57 @@ +No,Latitude,Longitude,Name,GCID +1,1.000000,2.000000,"WPT001",1 +2,1.000000,2.000000,"WPT002",1 +3,1.000000,2.000000,"WPT003",1 +4,1.000000,2.000000,"WPT004",1 +5,1.000000,2.000000,"WPT005",16 +6,1.000000,2.000000,"WPT006",256 +7,1.000000,2.000000,"WPT007",4096 +8,1.000000,2.000000,"WPT008",20480 +9,1.000000,2.000000,"WPT009",65535 +10,1.000000,2.000000,"WPT010",65536 +11,1.000000,2.000000,"WPT011",512401 +12,1.000000,2.000000,"WPT012",14365216 +13,1.000000,2.000000,"WPT013",27294510 +14,1.000000,2.000000,"WPT014",27294515 +15,1.000000,2.000000,"WPT015",28218030 +16,1.000000,2.000000,"WPT016",887092560 +17,1.000000,2.000000,"WPT017",27512202990 +18,1.000000,2.000000,"WPT018",852890626320 +19,1.000000,2.000000,"WPT019",26439621749550 +20,1.000000,2.000000,"WPT020",819628286569680 +21,1.000000,2.000000,"WPT021",25408476895993710 +22,1.000000,2.000000,"WPT022",787662783788138640 +23,1.000000,2.000000,"WPT023", +24,1.000000,2.000000,"WPT024", +25,1.000000,2.000000,"WPT025", +26,1.000000,2.000000,"WPT026", +27,1.000000,2.000000,"WPT027", +28,1.000000,2.000000,"WPT028", +29,1.000000,2.000000,"WPT029", +30,1.000000,2.000000,"WPT030", +31,1.000000,2.000000,"WPT031", +32,1.000000,2.000000,"WPT032", +33,1.000000,2.000000,"WPT033", +34,1.000000,2.000000,"WPT034", +35,1.000000,2.000000,"WPT035", +36,1.000000,2.000000,"WPT036", +37,1.000000,2.000000,"WPT037", +38,1.000000,2.000000,"WPT038", +39,1.000000,2.000000,"WPT039", +40,1.000000,2.000000,"WPT040", +41,1.000000,2.000000,"WPT041", +42,1.000000,2.000000,"WPT042", +43,1.000000,2.000000,"WPT043", +44,1.000000,2.000000,"WPT044", +45,1.000000,2.000000,"WPT045", +46,1.000000,2.000000,"WPT046", +47,1.000000,2.000000,"WPT047", +48,1.000000,2.000000,"WPT048", +49,1.000000,2.000000,"WPT049", +50,1.000000,2.000000,"WPT050", +51,1.000000,2.000000,"WPT051", +52,1.000000,2.000000,"WPT052", +53,1.000000,2.000000,"WPT053", +54,1.000000,2.000000,"WPT054",12345 +55,1.000000,2.000000,"WPT055",123456 +56,1.000000,2.000000,"WPT056",787662783788138640 diff --git a/testo.d/unicsv.test b/testo.d/unicsv.test index 96140a437..03383562a 100644 --- a/testo.d/unicsv.test +++ b/testo.d/unicsv.test @@ -15,6 +15,9 @@ gpsbabel -i gpx -f ${REFERENCE}/gc/GC7FA4.gpx -o unicsv,utc=0 -F ${TMPDIR}/gcuni gpsbabel -i unicsv,utc=0 -f ${REFERENCE}/gc/GC7FA4~unicsv.csv -o unicsv,utc=0 -F ${TMPDIR}/gcunicsv-2.csv compare ${TMPDIR}/gcunicsv-1.csv ${TMPDIR}/gcunicsv-2.csv +gpsbabel -i unicsv -f ${REFERENCE}/gc/gcid-input.csv -o unicsv -F ${TMPDIR}/gcid-output.csv +compare ${REFERENCE}/gc/gcid-output.csv ${TMPDIR}/gcid-output.csv + # check header detection features gpsbabel -i unicsv,utc=0 -f ${REFERENCE}/headerdetection.unicsv -x transform,trk=wpt -o gpx,garminextensions -F ${TMPDIR}/headerdetection~unicsv.gpx compare ${REFERENCE}/extensiondata~unicsv.gpx ${TMPDIR}/headerdetection~unicsv.gpx diff --git a/unicsv.cc b/unicsv.cc index cd0758499..50ed4e6bb 100644 --- a/unicsv.cc +++ b/unicsv.cc @@ -323,23 +323,61 @@ static QVector unicsv_args = { /* helpers */ -// There is no test coverage of this and it's been wrong for years and -// nobody has noticed... -static int -unicsv_parse_gc_id(const QString& str) +// Parse GC-Code / geo cache reference code into int64 (GC-ID) +// (see also https://api.groundspeak.com/documentation#referencecodes) +static long long +unicsv_parse_gc_code(const QString& str) { - int res = 0; - const QString kBase35 = "0123456789ABCDEFGHJKMNPQRTVWXYZ"; // ILOSU are omitted. - if (str.startsWith("GC")) { - int base35 = str.size() > 6; // above GCFFFF? - QString s = str.mid(2); - while (!s.isEmpty()) { - res = res * 16 + kBase35.indexOf(s[0]); - s = str.mid(1); + if (! str.startsWith("GC")) { + return 0; + } + + // Remove "GC" prefix + QString s = str.mid(2); + // Replacements according to groundspeak api documentation + s.replace('S', '5'); + s.replace('O', '0'); + // Remove leading zeros. some online converters do that as well. + while (s.startsWith('0')) { + s.remove(0, 1); + } + + // We have these cases: + // * 1-3 digits => base 16 + // * 4 digits, first one is 0-F => base 16 + // * 4 digits, first one G-Z => base 31 + // * 5-12 digits => base 31 + // * 13- digits => exceeds int64_t + // + int base; + const QString kBase31 = "0123456789ABCDEFGHJKMNPQRTVWXYZ"; // ILOSU are omitted. + if (s.size() >= 1 && s.size() <= 3) { + base = 16; + } else if (s.size() == 4) { + if (kBase31.indexOf(s[0]) < 16) { + base = 16; + } else { + base = 31; } - if (base35) { - res -= 411120; + } else if (s.size() >= 5 && s.size() <= 12) { + base = 31; + } else { + return 0; + } + + long long res = 0; + for (auto c : qAsConst(s)) { + int val = kBase31.indexOf(c); + if (val < 0 || (base == 16 && val > 15)) { + return 0; } + res = res * base + val; + } + if (base == 31) { + res -= 411120; + } + if (res < 0) { + res = 0; } return res; } @@ -1010,9 +1048,13 @@ unicsv_parse_one_line(const QString& ibuf) switch (unicsv_fields_tab[column]) { case fld_gc_id: - gc_data->id = value.toInt(); - if (gc_data->id == 0) { - gc_data->id = unicsv_parse_gc_id(value); + // First try to decode as numeric GC-ID (e.g. "575006"). + // If that doesn't succedd, try to decode as GC-Code + // (e.g. "GC1234G"). + bool ok; + gc_data->id = value.toLongLong(&ok, 10); + if (!ok) { + gc_data->id = unicsv_parse_gc_code(value); } break; case fld_gc_type: diff --git a/xmldoc/formats/unicsv.xml b/xmldoc/formats/unicsv.xml index ba2884778..333baf0de 100644 --- a/xmldoc/formats/unicsv.xml +++ b/xmldoc/formats/unicsv.xml @@ -32,7 +32,7 @@ exported = Geocache export date found = Geocache last found date fix = 3d, 2d, etc. - gcid = Geocache cache id + gcid = Geocache cache id. This accepts GC-ID ("575006") and GC-Code ("GC1234G"). geschw = Geschwindigkeit (speed) hdop = Horizontal dilution of precision head = Heading / Course true -- 2.30.2